var http = require('http');
var sys = require("sys");
var fs = require("fs");

var object = null;
var tasks = [];
var cur_url;
function get_caption(url) {
	var newurl = url.replace("http://", '').replace("https://", '').replace("/", '');

	var options = {
		host : newurl
		// host:'www.eygle.com'
	};

	console.log(options.host);

	var html = '';
	console.log("http.get");
	http.get(options, function(res) {
		res.on('data', function(data) {
			html += data;
		}).on('end', function() {
			//*
			var caption = html.match(/<title>.*<\/title>/);
			if (null != caption && caption.length) {
				caption = caption[0].replace(/<title>/, '').replace(/<\/title>/, '');
			}
			console.log(caption);
			// sys.puts(caption);
			new_caption(caption);
			next();
		});
	}).on('error', function(e) {
		console.log('error------------------------------------------------error');
		console.log('problem with request: ' + e.message);
		next();
	});
}

function start_getting() {
	console.log("start_getting");
	next();
}

function new_caption(caption) {
	console.log("new_caption:" + caption);
	object[cur_url].caption = caption;
}

function next() {
	console.log("next");
	if (tasks.length > 0) {
		cur_url = tasks.pop();
		get_caption(cur_url);
	} else {
		// finish();
	}
}

function finish() {
	console.log("finish");
	// return;
	console.log(object);
	newData = JSON.stringify(object);
	// console.log(newData);
	fs.writeFile('../datas/urls_withCaption.json', newData, function(err) {
		//fs.writeFile('urls_withCaption.json', "abcdefg", function(err) {
		console.log('err:' + err);
		if (err)
			throw err;
		console.log('has finished');
	});
}

function start() {
	fs.readFile("../datas/urls.json", function(err, data) {
		if (err) {
			sys.puts("err");
			throw err;
		}
		sys.puts("File is OK!");
		object = eval('(' + data + ')');

		for (var p in object) {
			if ("count" != p) {
				tasks.push(p);
				//object[p]
			}
		}
		start_getting();
		// finish();
	});
}

function test() {
	html = '';
	options = "http://www.google.com.hk/favicon.ico";
	http.get(options, function(res) {
		res.on('data', function(data) {
			console.log('data');
			html += data;
		}).on('end', function() {
			console.log('end');
			console.log(html);	
			return;		
			fs.writeFile('favicon.ico', html, function(err) {
				//fs.writeFile('urls_withCaption.json', "abcdefg", function(err) {
				console.log('err:' + err);
				if (err)
					throw err;
				console.log('has finished');
			});
		});
	}).on('error', function(e) {
		console.log('Got that pesky error trapped');
	});
}

// start();test();
